unit Proses_Data;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, ZDataset, ZConnection, DB, ZAbstractRODataset, ZAbstractDataset,
  ZAbstractTable, StdCtrls, Unit_External,Message_Form, Setting;

type

  tpenyimpan = class(TObject)
    id : integer;
    nama        : string;
  end;
  TDataPemilih = class(TObject)
    id : string;
    nama : string;
    namaibu : string;
    tempat : string;
    tanggal : string;
    kelamin : string;
  end;
  tabelpemilih = class(TObject)
    index : array of tdatapemilih;
  end;
  TForm_Data = class(TForm)
    ZTable1: TZTable;
    ZConnection1: TZConnection;
    ZQuery1: TZQuery;

    ZQuery2: TZQuery;
    ZQuery3: TZQuery;
    ZTable2: TZTable;
Button1: TButton;
    Label1: TLabel;
    Button2: TButton;
    procedure FormCreate(Sender: TObject);
    procedure FormClose(Sender: TObject; var Action: TCloseAction);
    procedure Button1Click(Sender: TObject);
    procedure Button2Click(Sender: TObject);

  end;
  TForm_Thread = class(TThread)
    ZTable1: TZTable;
    Label1: TLabel;
    ZConnection1: TZConnection;
    ZQuery1: TZQuery;

    ZQuery2: TZQuery;
    ZQuery3: TZQuery;
    ZTable2: TZTable;

    procedure Execute; override;
    procedure ProsesDataGanda(id:string;nama:string;ibu:string;tempat:string;tanggal:string;kelamin:string);
//    function getArray(dict:TDictionary<string,tstringlist>;value:string):Tstringlist;
    function mergeArray(array1:TStringList;array2:TStringList;operation:string):TStringList;
    function RemoveDuplicates( stringList : TStringList) :TStringList;
    procedure test();
    function NormalizeStringList(S: TStringList): TStringList;
    procedure test2();
    procedure mergeor(A,B,C:TStringList);
    procedure mergeand(A,B,C: TStringList);
    procedure mergeand2(A,B,C: TStringList);
    procedure prosesDatabaseGanda(id:string;nama:string;ibu:string;tempat:string;tanggal:string;kelamin:string;input:TStringList;level:integer);
    function getNamaPenyimpan(id:integer):string;
    function getDataPemilih(id:string):TDataPemilih;
    procedure compareDataGanda(id,id2,nama,nama2,ibu,ibu2,tempat,tempat2,tanggal,tanggal2,kelamin,kelamin2:string;level:integer);
    function isDataCompared(id,id2:string):bool;
    procedure reduce(A,B,C:Tstringlist);
    procedure initialize();
  private
    { Private declarations }
  public
    { Public declarations }
    finish:boolean;

    currentitem:integer;
    totalitem:integer;
    currentFile:string;
    messageForm:TMessageForm;
    ext:ExternalFile;
    queryRapel:TStringList;
    tabelPenyimpan:array of tpenyimpan;
//    buffDictNama, buffDictInisial, buffDictIbu, buffDictInisialIbu, buffDictTanggal, buffDictTempat : TDictionary<String,TStringList>;
//    dictDataPemilih : TDictionary<string,TDataPemilih>;
{    dictCompare : TDictionary<string,string>;
    dictAllPemilih : TDictionary<string,TDictionary<string,tdatapemilih>>;
    dictGanda:Tdictionary<string,tstringlist>;}

  end;

var
  Form_Data: TForm_Data;
    Thread:TForm_Thread;

implementation

{$R *.dfm}

procedure TForm_Thread.Execute();
var i,j,count:integer;
  query:string;
  startTime, endTime, elapsedTime: TDateTime;
  mfile:textfile;
  pemilih:tstringlist;
  key:string;
  datapemilih:TDataPemilih;
  allpemilih:tabelpemilih;
//  dictpemilih:tdictionary<string,tdatapemilih>;

begin
  finish:=false;
  currentitem:=0;
  //Showmessage('Proses akan memakan waktu yang cukup lama, mohon tunggu beberapa saat tanpa melakukan sesuatu terhadap program.');
  initialize();
  queryRapel:=TStringList.Create;
  count:=0;
  totalitem:=Length(tabelPenyimpan);
  for I := 0 to Length(tabelPenyimpan)-1 do
  begin
    currentitem:=currentitem+1;
    currentFile:=tabelpenyimpan[I].nama;
    Label1.Caption:='Current item : '+currentFile+'. Processing item '+inttostr(currentitem)+'/'+inttostr(totalitem);
    //ShowMessage('tabel '+tabelpenyimpan[I].nama);
{    if (dictAllPemilih.TryGetValue(tabelpenyimpan[I].nama, dictpemilih) = True) then
    begin
      for Key in dictpemilih.Keys do
      begin
        count:=count+1;
        ProsesDataGanda(inttostr(tabelPenyimpan[I].id)+'-'+
        dictpemilih.Items[key].id,dictpemilih.Items[key].nama,
        dictpemilih.Items[key].namaibu, dictpemilih.Items[key].tempat,
        dictpemilih.Items[key].tanggal, dictpemilih.Items[key].kelamin);
      end;
    end;
 } end;
  if(queryRapel.Count>0)then
  begin
      ZQuery2.SQL.Clear;
      query:='';
      for I := 0 to queryRapel.Count - 1 do
      begin
        query:=query+queryRapel[I];
      end;

      ZQuery2.SQL.Text:='INSERT IGNORE INTO data_ganda (ID_USER1,ID_USER2,KETERANGAN,VERIFIKASI)'
      +' VALUES '+query;
      ZQuery2.ExecSQL;
  
  end;

{  for Key in dictganda.Keys do
   begin
      query:='';
      zquery1.Active:=false;
      zquery1.sql.Clear;
      for I := 0 to dictganda.Items[key].Count - 1 do
        begin
          if(I<>0)then
            query:=query+',';
          query:=query+''+dictganda.Items[key][I];
        end;
      ZQuery1.SQL.Text:='UPDATE '+key+' SET ganda = "1" WHERE ID_USER IN ('+query+')';
      zquery1.ExecSQL;
   end;
 } endTime := Now();
  elapsedTime := endTime - startTime;
  ShowMessage(inttostr(count)+' '+FormatDateTime('hh:nn:ss.zzz',elapsedTime));
{  FreeAndNil(dictpemilih);
  FreeAndNil(dictDataPemilih);
  FreeAndNil(dictCompare);
  FreeAndNil(dictAllPemilih);
  FreeAndNil(dictGanda);
 } finish:=true;

end;
procedure TForm_Thread.ProsesDataGanda(id:string;nama:string;ibu:string;tempat:string;tanggal:string;kelamin:string);
var myfile:textfile;
  array1,array2,array3,array4,array5,array6:Tstringlist;
  murnilv3,murnilv1, murnilv2, garray1, garray2, garraylv3, garray3, garray4, garraylv2:TStringlist;
  startTime, endTime, elapsedTime: TDateTime;
  I: Integer;
  debug:bool;
begin
  debug:=false;

{  array1:=getArray(buffDictNama, ext.getNama(nama));
  array2:=getArray(buffDictInisial, ext.getInisial(nama));
  array3:=getArray(buffDictIbu, ext.getIbu(ibu));
  array4:=getArray(buffDictInisialIbu, ext.getInisialIbu(ibu));
  array5:=getArray(buffDictTanggal, ext.getTanggal(tanggal));
  array6:=getArray(buffDictTempat, ext.getTempat(tempat));

 }
  garray1:=tstringlist.Create;
  garray2:=tstringlist.Create;
  garraylv3:=tstringlist.Create;
  garraylv2:=tstringlist.Create;

  murnilv1:=tstringlist.Create;
  murnilv2:=tstringlist.Create;
  murnilv3:=tstringlist.Create;
  try
//startTime := Now();

    if(ibu='')then
    begin
        mergeor(array1,array2,garray1); // nama
        mergeand2(garray1,array5,garraylv3); // nama and tanggal
        reduce(garraylv3, murnilv2, murnilv3);
        mergeand2(garraylv3, array6, murnilv2); // nama and tanggal and tempat
        if(debug) then Showmessage('tanpa ibu');
    end
    else
    begin
      mergeor(array1,array2,garray1); // nama
      mergeor(array3,array4,garray2); // ibu
      mergeand2(garray1,garray2,garraylv3); // nama and ibu
      mergeand2(garraylv3, array5, garraylv2); // nama and ibu and tanggal
      reduce(garraylv3, garraylv2, murnilv3);
      murnilv3:=TStringList.Create;
      mergeand2(garraylv2, array6, murnilv1); // nama and ibu and tanggal and tempat
      reduce(garraylv2, murnilv1, murnilv2);
    end;

//
//    endTime := Now();
//    elapsedTime := endTime - startTime;
//    ShowMessage(FormatDateTime('hh:nn:ss.zzz',elapsedTime));

    prosesDatabaseGanda(id,nama,ibu,tempat,tanggal,kelamin,murnilv1, 1);
    prosesDatabaseGanda(id,nama,ibu,tempat,tanggal,kelamin,murnilv2, 2);
    prosesDatabaseGanda(id,nama,ibu,tempat,tanggal,kelamin,murnilv3, 3);

//    startTime := Now();

   if(debug)then
    begin
      ShowMessage('lv3 '+IntToStr(murnilv3.Count));
      ShowMessage('lv2 '+IntToStr(murnilv2.Count));
      ShowMessage('lv1 '+IntToStr(murnilv1.Count));
    end;

    if(debug)then
    begin
      AssignFile(myfile, 'test.txt');
      rewrite(myfile);
      for I := 0 to garray1.Count-1 do
      begin
        writeln(myfile,garray1[I]);
      end;
      writeln(myfile,'garray1----------------');
      for I := 0 to garray2.Count-1 do
      begin
        writeln(myfile,garray2[I]);
      end;
      writeln(myfile,'garray2----------------');
      for I := 0 to garraylv3.Count-1 do
      begin
        writeln(myfile,garraylv3[I]);
      end;
      writeln(myfile,'garraylv3----------------');
      for I := 0 to garraylv2.Count-1 do
      begin
        writeln(myfile,garraylv2[I]);
      end;
      writeln(myfile,'garraylv2----------------');

      for I := 0 to murnilv3.Count-1 do
      begin
        writeln(myfile,murnilv3[I]);
      end;
      writeln(myfile,'murnilv3----------------');
      for I := 0 to murnilv2.Count-1 do
      begin
        writeln(myfile,murnilv2[I]);
      end;
      writeln(myfile,'murnilv2----------------');
      for I := 0 to murnilv1.Count-1 do
      begin
        writeln(myfile,murnilv1[I]);
      end;
      writeln(myfile,'murnilv1----------------');


      CloseFile(myfile);
    end;
  finally
    array1:=tstringlist.Create;
    array2:=tstringlist.Create;
    array3:=tstringlist.Create;
    array4:=tstringlist.Create;
    array5:=tstringlist.Create;
    array6:=tstringlist.Create;
    garray1:=tstringlist.Create;
    garray2:=tstringlist.Create;
    garraylv3:=tstringlist.Create;
    garraylv2:=tstringlist.Create;

    murnilv1:=tstringlist.Create;
    murnilv2:=tstringlist.Create;
    murnilv3:=tstringlist.Create;
    FreeAndNil(array1);
    FreeAndNil(array2);
    FreeAndNil(array3);
    FreeAndNil(array4);
    FreeAndNil(array5);
    FreeAndNil(array6);
    FreeAndNil(garray1);
    FreeAndNil(garray2);
    FreeAndNil(garraylv3);
    FreeAndNil(garraylv2);
    FreeAndNil(murnilv1);
    FreeAndNil(murnilv2);
    FreeAndNil(murnilv3);
  end;


end;
procedure TForm_Data.Button1Click(Sender: TObject);
var setting:TSetting_Form;
begin
  setting:=TSetting_Form.Create;
  ZConnection1.HostName:=setting.host;
  ZConnection1.Protocol:=setting.protocol;
  ZConnection1.Database:=setting.database;
  ZConnection1.User:=setting.user;
  ZConnection1.Password:=setting.password;
  ZConnection1.Connected:=true;
  ztable1.Connection:=ZConnection1;
  ZQuery1.Connection:=ZConnection1;
  ztable2.Connection:=ZConnection1;
  ZQuery2.Connection:=ZConnection1;



  if Thread=nil then
  begin
    Thread:=TForm_Thread.Create(false);
    Thread.ZConnection1:=ZConnection1;
    Thread.ZTable1:=ZTable1;
    Thread.ZTable2:=ZTable2;
    Thread.ZQuery1:=ZQuery1;
    Thread.ZQuery2:=ZQuery2;
    Label1.Caption:='Start';
    Thread.Label1:=label1;



  end;
//  if(Thread.finish) then
//  Showmessage('selesai')
//  else showmessage('belum');



end;

procedure TForm_Data.Button2Click(Sender: TObject);
begin
  ShowMessage('Indexing Berhasil Dilakukan');
end;

procedure TForm_Data.FormClose(Sender: TObject; var Action: TCloseAction);
begin
  Action := caFree;

end;

procedure TForm_Data.FormCreate(Sender: TObject);
begin
  //ext:=ExternalFile.Create;

end;
procedure TForm_Thread.initialize();
var A,B,C,D:TStringList;
  counter, counterpemilih: Integer;
  temp:Tpenyimpan;
  test,key,query:string;
  datapemilih:TDataPemilih;
  I: Integer;
  allpemilih:tabelpemilih;
//  dictpemilih:tdictionary<string,tdatapemilih>;
  testlist:tstringlist;
begin

{  dictDataPemilih := tdictionary<string,TDAtapemilih>.Create;
  dictAllPemilih := tdictionary<string,tdictionary<string,tdatapemilih>>.Create;
  dictCompare:=tdictionary<string,string>.create;
  dictganda:=tdictionary<string,tstringlist>.create;
 } testlist:=tstringlist.Create;

{  buffdictNama:=ext.getListContent('Indexing/nama_lengkap.txt');
  buffdictIbu:=ext.getListContent('Indexing/ibu_kandung.txt');
  buffdictInisial:=ext.getListContent('Indexing/inisial_nama.txt');
  buffdictInisialIbu:=ext.getListContent('Indexing/inisial_ibu.txt');
  buffdictTanggal:=ext.getListContent('Indexing/tanggal_lahir.txt');
  buffdictTempat:=ext.getListContent('Indexing/tempat_lahir.txt');
 }
  ZTable1.Active:=false;
  ZTable1.TableName:='penyimpan';
  ZTable1.Active:=true;
  SetLength(tabelPenyimpan, ZTable1.RecordCount);

  ZTable1.First;
  counter:=0;
  counterpemilih:=0;
  while (not ZTable1.Eof) do
  begin
    temp:= tpenyimpan.Create;
    temp.id:=strtoint(ZTable1.FieldByName('ID').AsString);
    temp.nama:=ZTable1.FieldByName('NAMA').AsString;
    tabelPenyimpan[counter]:=temp;

    counter := counter+1;

    ZTable2.Active:=false;
    ZTable2.TableName:=ZTable1.FieldByName('NAMA').AsString;
    ZTable2.Active:=true;
    ZTable2.first;

//    dictpemilih:=tdictionary<string,tdatapemilih>.create;
    while(not ztable2.Eof)do
    begin
      datapemilih:=TDataPemilih.Create;
      datapemilih.id:=ztable2.FieldByName('ID_USER').AsString;
      datapemilih.nama:=ztable2.FieldByName('NAMA').AsString;
      datapemilih.namaibu:=ztable2.FieldByName('IBU').AsString;
      datapemilih.tempat:=ztable2.FieldByName('TEMPAT').AsString;
      datapemilih.tanggal:=ztable2.FieldByName('TANGGAL').AsString;
      datapemilih.kelamin:=ztable2.FieldByName('KELAMIN').AsString;
//      dictpemilih.Add(ztable2.FieldByName('ID_USER').AsString, datapemilih);

      datapemilih:=TDataPemilih.Create;

      ztable2.next;
    end;
//    dictAllPemilih.Add(ZTable1.FieldByName('NAMA').AsString,dictpemilih);

    ZTable1.Next;
  end;
end;
{function TForm_Thread.getArray(dict:TDictionary<string,Tstringlist>; value:string):Tstringlist;
var mfile:TextFile;
    location:string;
    text:string;
    I: Integer;
    found:bool;
begin
  result:=TStringlist.Create;
  if (dict.TryGetValue(value, result) = True) then
    begin
    end
  else
      result:=TStringlist.Create;

end;
}
function TForm_Thread.getDataPemilih(id: string):TDataPemilih;
begin
  result:= TDataPemilih.Create;
{  if (dictDataPemilih.TryGetValue(id, result) = True) then
  begin

  end
  else
 }   result:= TDataPemilih.Create;
end;
function TForm_Thread.getNamaPenyimpan(id:integer):string;
var
  I: Integer;
begin
  result:='';

  for I := 0 to length(tabelpenyimpan)-1 do
  begin
    if(id=tabelpenyimpan[I].id)then
    begin
      result:=tabelpenyimpan[I].nama;
      break;
    end;
  end;

end;
procedure TForm_Thread.prosesDatabaseGanda(id:string;nama:string;ibu:string;tempat:string;tanggal:string;kelamin:string;input:Tstringlist;level:integer);
var listid,idsplit:TStringList;
  I,j,k: Integer;
  namatabel, nama2,ibu2,tempat2,tanggal2,kelamin2:string;
  datapemilih:TDataPemilih;
  startTime, endTime, elapsedTime: TDateTime;
//  dictpemilih:tdictionary<string,tdatapemilih>;

begin
  listid:=Tstringlist.Create;
  idsplit:=TStringList.Create;
  datapemilih:=getDataPemilih(id);
  if(datapemilih.nama='')then
  begin
    datapemilih.nama := nama;
    datapemilih.namaibu:=ibu;
    datapemilih.tempat:=tempat;
    datapemilih.tanggal:=tanggal;
    datapemilih.kelamin:=kelamin;
    datapemilih.id:=id;
//    dictDataPemilih.AddOrSetValue(id,datapemilih);
    datapemilih:=TDataPemilih.Create; // dibersihkan
  end;
  try
  //  ExtractStrings([','], [], PChar(input), listid);
    listid.Assign(input);
  //  ShowMessage(IntToStr(listid.Count));
    for I := 0 to listid.Count - 1 do
    begin
    //ShowMessage(IntToStr(I));
      if(listid[I]<>id)then
      begin
        if(not isDataCompared(id,listid[I]))then
        begin
          idsplit:=TStringList.Create;
          datapemilih:=getDataPemilih(listid[i]);
          if(datapemilih.nama='')then
          begin
              ExtractStrings(['-'], [], PChar(listid[I]), idsplit);
              namatabel := getNamaPenyimpan(strtoint(idsplit[0]));
              if(namatabel<>'')then
              begin
{                if (dictallpemilih.TryGetValue(namatabel, dictpemilih) = True) then
                begin
                  if (dictpemilih.TryGetValue(idsplit[1], datapemilih) = True) then
                  begin
                      dictDataPemilih.Add(listid[i],datapemilih);
                      nama2:=datapemilih.nama;
                      ibu2:=datapemilih.namaibu;
                      tempat2:=datapemilih.tempat;
                      tanggal2:=datapemilih.tanggal;
                      kelamin2:=datapemilih.kelamin;
//                      ShowMessage(nama2+ibu2+tempat2+tanggal2+kelamin2);
                      if(kelamin=kelamin2)then
                        compareDataGanda(id, listid[I], nama,nama2,ibu,ibu2,tempat,tempat2,tanggal,tanggal2,kelamin,kelamin2,level);
                  end;


                end;
 }             end;
          end
          else //  data pemilih sudah ada
          begin
//            ShowMessage('udah ada');
            nama2:=datapemilih.nama;
            ibu2:=datapemilih.namaibu;
            tempat2:=datapemilih.tempat;
            tanggal2:=datapemilih.tanggal;
            kelamin2:=datapemilih.kelamin;
            if(kelamin=kelamin2)then
              compareDataGanda(id, listid[I], nama,nama2,ibu,ibu2,tempat,tempat2,tanggal,tanggal2,kelamin,kelamin2,level);
          end;
        end;
      end;
    end;
  finally
    FreeAndNil(listid);
    FreeAndNil(idsplit);
  end;
end;
procedure TForm_Thread.reduce(A,B,C:Tstringlist);
var i,pos:integer;
begin
  C.Duplicates:=dupIgnore;
  C.Sorted:=true;
  C.BeginUpdate;
  for I := 0 to A.Count - 1 do
  begin
    pos:=B.IndexOf(A[i]);
//    ShowMessage(inttostr(i)+' - '+B[I]+' ketemu '+inttostr(A.IndexOf(B[i])));
    if(pos=-1)then
    begin
//    sl.Add(B[I]);
      C.Add(A[I]);
//      D.Delete(pos);
    end;
//    else
//      D.Add(B[I]);

  end;
  C.EndUpdate;

end;
function TForm_Thread.isDataCompared(id,id2:string):bool;
var temp:string;
begin
  result:=false;
{  if (dictCompare.TryGetValue(id, temp) = True) then
  begin
    result:=true;
  end
  else
  begin
     if (dictCompare.TryGetValue(id2, temp) = True) then
      begin
        dictCompare.Add(id,id2);
        result:=true;
      end
      else
      begin
        dictCompare.Add(id,id2);
        dictCompare.Add(id2,id);
      end;
  //end;
 } if(result)then
  begin
//    showmessage('wess sudah ada, dict compare:'+inttostr(dictCompare.Count)+', dict data:'+inttostr(dictDataPemilih.Count));

  end;
end;
procedure TForm_Thread.compareDataGanda(id,id2,nama,nama2,ibu,ibu2,tempat,tempat2,tanggal,tanggal2,kelamin,kelamin2:string;level:integer);
var k,i:integer;
    ganda:bool;
    query,key,namatabel:string;
    A,idsplit:TStringList;
begin
//  ShowMessage('COMPARE '+id+' '+id2);
  ganda:=false;
  if(ibu='')then
  begin
      if((level=3)and(nama=nama2)and(tanggal=tanggal2))then
      begin
        ganda:=true;
      end
      else if((level=2)and(nama=nama2)and(tanggal=tanggal2)and(tempat=tempat2))then
      begin
        ganda:=true;
      end
  end
  else
  begin
      if((level=3)and((nama=nama2)or(ibu=ibu2)))then
      begin
        ganda:=true;
      end
      else if((level=2)and((nama=nama2)or(ibu=ibu2))and(tanggal=tanggal2))then
      begin
        ganda:=true;
      end
      else if((level=1)and((nama=nama2)or(ibu=ibu2))and(tanggal=tanggal2)and(tempat=tempat2))then
      begin
        ganda:=true;
      end;
  end;
  if(ganda)then
  begin
//        ShowMessage('query masuk');
//    showmessage(inttostr(dictganda.count));
    idsplit:=tstringlist.Create;
    ExtractStrings(['-'], [], PChar(id), idsplit);
    namatabel := getNamaPenyimpan(strtoint(idsplit[0]));
    if(namatabel<>'')then
    begin
{      if (dictganda.TryGetValue(namatabel, A) = True) then
      begin
        A.Add(idsplit[1]);
      end
      else
      begin
        A:=Tstringlist.Create;
        A.Add(idsplit[1]);
        dictganda.Add(namatabel,A);
      end;}
    end;
    idsplit:=tstringlist.Create;
    ExtractStrings(['-'], [], PChar(id2), idsplit);
    namatabel := getNamaPenyimpan(strtoint(idsplit[0]));
    if(namatabel<>'')then
    begin
{      if (dictganda.TryGetValue(namatabel, A) = True) then
      begin
        A.Add(idsplit[1]);
      end
      else
      begin
        A:=Tstringlist.Create;
        A.Add(idsplit[1]);
        dictganda.Add(namatabel,A);
      end;
 }   end;

    query:='';
    if(queryRapel.Count>0)then
      query:=',';
    query:=query+'('+QuotedStr(id)+','+QuotedStr(id2)+','+QuotedStr(IntToStr( level))+','+QuotedStr('0')+')';
    queryRapel.Add(query);
    if(queryRapel.Count>999)then
    begin
//      ShowMessage('udah seribu'+query);
        ZQuery2.SQL.Clear;
        query:='';
        for k := 0 to queryRapel.Count - 1 do
        begin
          query:=query+queryRapel[k];
        end;

        ZQuery2.SQL.Text:='INSERT IGNORE INTO data_ganda (ID_USER1,ID_USER2,KETERANGAN,VERIFIKASI)'
        +' VALUES '+query;
        ZQuery2.ExecSQL;
        queryRapel.Clear;
    end;

  end;
end;
function TForm_Thread.mergeArray(array1:TStringList;array2:TStringList;operation:string):TStringList;
var return,B,C:TStringList;
    arrayiterator1:TStringList;
    arrayiterator2:TStringList;

    finish,debug:bool;
    I,J, K:integer;

begin
  return:= TStringList.Create;


  if(operation<>'')then
  begin

    B:= TStringList.Create;
    C:= TStringList.Create;
    operation:=LowerCase(operation);

    finish:=false;
    debug:=true;
    I:=0;
    if(array1.Count<array2.Count)then
      begin
        arrayiterator1:=array1;
        arrayiterator2:=array2;
      end
    else
      begin
        arrayiterator1:=array2;
        arrayiterator2:=array1;
      end;
    J:=arrayiterator2.Count-1;
    ExtractStrings(['-'], [], PChar(arrayiterator2[J]), B);

    while ((I<arrayiterator1.Count)and(not finish) and (J>=0)) do
    begin
      C.Clear;
      ExtractStrings(['-'], [], PChar(arrayiterator1[I]), C);
      if(debug) then ShowMessage(B[0]+' vs '+C[0]);

      if(StrToInt(C[0])>StrToInt(B[0]))then // langsung masukkan semua, si B yang paling tinggi aja lebih kurang dari C
      begin
        return.Add(arrayiterator2[K]);
        // masukkan sisanya
        while(I<arrayiterator1.Count) do
        begin
          return.Add(arrayiterator1[I]);
          I:=I+1;
        end;
        finish:=true;
        break; // stop
      end
      else if(StrToInt(B[0])=StrToInt(C[0]))then //sama masukkan salah satu ke hasil
      begin
        if(debug) then ShowMessage('B0 sama C0');
        if(debug) then ShowMessage(B[1]+' vs '+C[1]);
        if(StrToInt(B[1])<StrToInt(C[1]))then // id usernya lebih kecil, masukkan
        begin
          if(debug) then ShowMessage(B[1]+' kecil dari '+C[1]);
          for K := 0 to J do
          begin
            return.Add(arrayiterator2[K]);
          end;
          // masukkan sisanya
          while(I<arrayiterator1.Count)do
          begin
            return.Add(arrayiterator1[i]);
            I:=I+1;
          end;
          finish:=true;
          break; // stop
        end
        else  // lebih besar atau sama dengan
        begin
          if(debug) then ShowMessage(B[1]+' lebih besar dari atau sama dengan '+C[1]);
          return.Add(arrayiterator1[I]);
          // lanjutkan iterasi
        end;
      end
      else // lebih besar, masukkan
      begin
        if(debug) then ShowMessage('B0 lebih dari atau sama dengan C0');
        return.Add(arrayiterator1[I]);

        // lanjutkan iterasi
      end;
      I:=I+1;
      if((I=array1.Count)and(not finish)and(J>=0))then // masukkan di ujung
      begin
        if(debug) then ShowMessage('masukkan di ujung');

        for K := 0 to J do
          begin
            return.Add(arrayiterator2[K]);
          end;

      end;
    end;

//    if(operation='and')then
//    begin
//
//    end
//    else if(operation='or')then
//    begin
//
//    end;
  end;
  ShowMessage(IntToStr(return.Count));
  result:=return;
end;

procedure TForm_Thread.mergeor(A,B,C: TStringList);
var
  I: Integer;
begin
//  C.Assign(A);
  C.Create;
  C.Sorted:=true;
//  C.Sorted:=true;
  C.Duplicates:=dupIgnore;
  C.BeginUpdate;
  for I := 0 to A.Count - 1 do
  begin
    C.Add(A[I]);
  end;
  for I := 0 to B.Count - 1 do
  begin
    C.Add(B[I]);
  end;
  C.EndUpdate;
  

end;

procedure TForm_Thread.mergeand(A,B,C: TStringList);
var
  I,loop,initial: Integer;
  temp,sl:Tstringlist;

  startTime, endTime, elapsedTime: TDateTime;
begin
  
//  C.Assign(A);
  C.Create;
//  C.Assign(A);
  C.Sorted:=true;
  C.Duplicates:=dupIgnore;
  C.BeginUpdate;

  temp:=Tstringlist.Create;
  temp.Assign(A);
  //temp.Sort;

                 
  temp.Duplicates:=dupError;
  temp.Sorted:=true;
  temp.BeginUpdate;
startTime := Now();

//
//
  for I := 0 to B.Count - 1 do
  begin
     try
        temp.Add(B[I]);
     except
          C.Add(B[I]);
     end;
  end;
  endTime := Now();
  elapsedTime := endTime - startTime;
//  ShowMessage(''+FormatDateTime('h:n:s.z', elapsedTime));

  C.EndUpdate;
  temp.EndUpdate;
  FreeAndNil(temp);
  

end;

procedure TForm_Thread.mergeand2(A,B,C:TStringlist);
var i,pos:integer;
//  initial,I,loop:integer;
//  sl,temp:TStringlist;
//  starttime, endtime: tdatetime;
begin
//  C:=Tstringlist.Create;
  C.Duplicates:=dupIgnore;
  C.Sorted:=true;
  C.BeginUpdate;

//    starttime := now;
//  sl:=TStringList.Create;
//  sl.Assign(A);

  for I := 0 to B.Count - 1 do
  begin
    pos:=A.IndexOf(B[i]);
//    ShowMessage(inttostr(i)+' - '+B[I]+' ketemu '+inttostr(A.IndexOf(B[i])));
    if(pos<>-1)then
    begin
//    sl.Add(B[I]);
      C.Add(B[I]);
//      D.Delete(pos);
    end
//    else
//      D.Add(B[I]);

  end;
  C.EndUpdate;
//  D.EndUpdate;
//  ShowMessage(IntToStr(C.Count));
//  initial:=sl.Count;
//  with sl do
//    begin
//      Sorted := True; //sort the list to prepare for fast dup check and removal
//      Duplicates := dupIgnore;
//    end;
//      
//    loop:=initial;
//    while loop>2 do
//    begin
//      if sl[loop-1][1]=sl[loop-2][1] then
//        begin
//        if sl[loop-1]=sl[loop-2] then
//         begin
////          C.Add(sl[loop-1]);
////         sl.Delete(loop-1);
//          temp.Add(sl[loop-2]);
//         end;
//        end;
//    Dec(loop);
//    end;
//    ShowMessage(FormatDateTime('nn:ss.zzz', Now()-StartTime));
//    ShowMessage(IntToStr(temp.Count));
//    for I := 0 to temp.Count - 1 do
//      ShowMessage(temp[I]);
    
end;


function TForm_Thread.RemoveDuplicates( stringList : TStringList) :TStringList;
 var
   buffer: TStringList;
   cnt: Integer;
 begin
//   stringList.Sort;
   result:=TStringList.Create;
//   result.Sorted:=true;
   buffer := TStringList.Create;
   try
     buffer.Sorted := True;
     buffer.Duplicates := dupIgnore;
     buffer.BeginUpdate;
     for cnt := 0 to stringList.Count - 1 do
       buffer.Add(stringList[cnt]) ;
     buffer.EndUpdate;
     result.Assign(buffer) ;
   finally
     FreeandNil(buffer) ;

   end;
 end;

 procedure TForm_Thread.test();
   var
   sl,sl2,outsl : TStringList;
   cnt : integer;
  I: Integer;
 begin
   Randomize;
   sl := TStringList.Create;
   sl2 := TStringList.Create;
   outsl:=tstringlist.Create;
   try
     for cnt := 1 to 1000000 do
       sl.Add('test'+IntToStr(Random(3))) ;
     for cnt := 1 to 1000000 do
       sl2.Add('test'+IntToStr(Random(3))) ;

     ShowMessage('With duplicates: ' + #13#10 + IntToStr(sl.Count)) ;

//     outsl:=RemoveDuplicates(sl) ;

//      outsl.Assign(sl);
//      for I := 0 to sl.Count - 1 do
//      begin
//        outsl.Add(sl[I]);
//      end;
//      outsl:=NormalizeStringList(sl);
              mergeand(sl,sl2,outsl);
//    mergeor(sl,sl2,outsl);
     ShowMessage('Without duplicates: ' + #13#10 + IntToStr(outsl.Count)) ;
     for I := 0 to outsl.Count - 1 do
        Showmessage(outsl[I]);
   finally
     sl.Free;
   end;
 end;
 procedure TForm_Thread.test2();
 var
   sl, outsl : TStringList;
   cnt : integer;
   Outstring: String;
  I: Integer;
 begin
   Randomize;
   //sl = stringlist with original values
   sl := TStringList.Create;
   try
      //fill initial stringlist with random test information
      for cnt := 1 to 10000000 do
         sl.Add(IntToStr(Random(2)));
      try
         //do not create outsl here, it is created in the function
         //assign the result from NormalizeStringList to outsl.
         //but make sure to free it below
         outsl := NormalizeStringList(sl);
//          mergeand(sl,sl2,outsl);
         ShowMessage('ganda '+IntToStr(outsl.Count));
         for I := 0 to outsl.Count - 1 do
          ShowMessage(outsl[I]);
//         ShowMessage('berikut');
//         for I := 0 to sl.Count - 1 do
//          ShowMessage(sl[I]);

         //outstring has your final output.
     finally
        //make sure you free outsl here, not in the function.
        if assigned(outsl) then
           freeandnil(outsl);
     end;
    finally
       sl.Free;
    end;
 end;
 function TForm_Thread.NormalizeStringList(S: TStringList): TStringList;
var
   hold: string;
   cnt, Index: Integer;
   temp:Tstringlist;
begin
   //create place holder for all duplicates in a 2nd stringlist
   result := TStringList.Create;
   try
      ShowMessage('masuk');
      //create the return stringlist
      result.Duplicates:=dupIgnore;
      //result.Sorted:=true;
      result.BeginUpdate;

      temp := TStringList.Create;
      temp.Duplicates := dupError;
      temp.Sorted:=true;
      temp.BeginUpdate;
      for cnt := 0 to S.Count - 1 do
      begin
         try
         //add 1st instance of a value into result
//      ShowMessage('1');
            temp.Add(S[cnt]);
         except
            //duplicates go into duplicate stringlist
            result.Add(S[cnt]);
//            ShowMessage('asd');
         end;
      end;
//      ShowMessage(IntToStr(duplicateBuffer.Count));
//      result.Sorted := False;
//      //parse through duplicate list,
//      //to modify result with number of instances
//      while DuplicateBuffer.Count > 0 do
//      begin
//         cnt := 1;
//         hold := duplicateBuffer[0];
//         duplicateBuffer.Delete(0);
//         while duplicateBuffer.Find(hold, Index) do
//         begin
//            inc(cnt);
//            duplicateBuffer.Delete(Index);
//         end;
//         if result.Find(hold, Index) then
//            result[Index] := result[Index]+'*'+IntToStr(cnt+1);
//      end;
      result.EndUpdate;
//      result.Assign(temp);
   finally
      FreeandNil(temp);
   end;
end;

end.
